
import warnings
import numpy as np                                            #Exploratory Data Analysis
import pyspark.pandas as ps
import seaborn as sns
import matplotlib.pyplot as plt
import plotly.graph_objects as go
from plotly.offline import iplot
from random import random
from pyspark.ml import Pipeline                                #Modeling
from pyspark.ml.regression import LinearRegression, DecisionTreeRegressor, RandomForestRegressor, GBTRegressor
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.evaluation import RegressionEvaluator
import mlflow                                                  #Deployment
import mlflow.sklearn
import mlflow.azureml
import azureml
from azureml.core import Workspace
from azureml.core.authentication import InteractiveLoginAuthentication
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score


dbutils.fs.mount(
  source = "wasbs://dataset@pricedatastorage.blob.core.windows.net",
  mount_point = "/mnt/data",
  extra_configs = {"fs.azure.account.key.pricedatastorage.blob.core.windows.net":"UUpMhyQ6jNC4fFy2QbYKaomV+hJAXB3P1oSxDP9jQyXIpBc1ljzZb4wAZQ4U7eyTYphlEHso+4tj+AStbx1GPQ=="})


df = spark.read.csv('/mnt/data/train.csv',inferSchema=True,header=True)
test_df = spark.read.csv('/mnt/data/test.csv',inferSchema=True,header=True)


df = df.to_pandas_on_spark()
df = df.to_pandas()


test_df = test_df.to_pandas_on_spark()
test_df = test_df.to_pandas()


df.display()


df.shape


df.describe()


df.info()


df.key = df.key.astype('datetime64[ns]')
df.pickup_datetime = df.pickup_datetime.astype('datetime64[ns]')
test_df.key = test_df.key.astype('datetime64[ns]')
test_df.pickup_datetime = test_df.pickup_datetime.astype('datetime64[ns]')


df.isna().sum().sort_values(ascending=False)


df.dropna(inplace=True)


df[df.fare_amount < 0].fare_amount


df.drop(df[df.fare_amount < 0].index, axis = 0, inplace = True)


df[df.passenger_count > 6]


df.drop(df[df.passenger_count > 6].index, axis = 0, inplace = True)


df.pickup_latitude.between(-90,90,inclusive=True).sum()


df.drop(df[df.pickup_latitude < -90].index, axis = 0, inplace = True) 
df.drop(df[df.pickup_latitude > 90].index, axis = 0, inplace = True)


df.dropoff_latitude.between(-90,90,inclusive=True).sum()
df.drop(df[df.dropoff_latitude < -90].index, axis = 0, inplace = True) 
df.drop(df[df.dropoff_latitude > 90].index, axis = 0, inplace = True)


df.pickup_longitude.between(-180,180,inclusive=True).sum()
df.drop(df[df.pickup_longitude < -180].index, axis = 0, inplace = True) 
df.drop(df[df.pickup_longitude > 180].index, axis = 0, inplace = True)


df.dropoff_longitude.between(-180,180,inclusive=True).sum()
df.drop(df[df.dropoff_longitude < -180].index, axis = 0, inplace = True) 
df.drop(df[df.dropoff_longitude > 180].index, axis = 0, inplace = True)


def distance(data, lat1, long1, lat2, long2):
    x = np.sin(np.radians(df[lat2]-df[lat1])/2.0) ** 2 + np.cos(np.radians(df[lat1])) * np.cos(np.radians(df[lat2])) * \
        np.sin(np.radians(df[long2]-df[long1])/2.0) ** 2
    y = 2 * np.arctan2(np.sqrt(x), np.sqrt(1 - x))
    data['distance'] = 3959 * y
    pass


distance(df, 'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude')


df['year'] = df.pickup_datetime.dt.year
df['month'] = df.pickup_datetime.dt.month
df['date'] = df.pickup_datetime.dt.day
df['day_of_week'] = df.pickup_datetime.dt.dayofweek
df['hour'] = df.pickup_datetime.dt.hour


Hist = [go.Histogram(x=df.fare_amount[:1000000])]
layout = go.Layout(title = 'Fare Amount')
fig = go.Figure(data = Hist, layout = layout)
iplot(fig)


plt.figure(figsize=(15,7))
plt.scatter(x=df.passenger_count, y=df.fare_amount)
plt.xlabel('Passenger Count')
plt.ylabel('Fare')


plt.figure(figsize=(15,7))
plt.scatter(x=df.date, y=df.fare_amount, s=1.5)
plt.xlabel('Date')
plt.ylabel('Fare')


plt.figure(figsize=(15,7))
plt.scatter(x=df.hour, y=df.fare_amount)
plt.xlabel('Hour')
plt.ylabel('Fare')


plt.figure(figsize=(15,7))
plt.scatter(x=df.day_of_week, y=df.fare_amount, s=1.5)
plt.xlabel('Day of Week')
plt.ylabel('Fare')


plt.figure(figsize=(16,10))
plt.scatter(x=df.distance, y=df.fare_amount, s=1.5)
plt.xlabel('Distance')
plt.ylabel('Fare')


fig, axs = plt.subplots(1, 2, figsize=(18,7))
axs[0].scatter(df.distance, df.fare_amount, alpha=0.2)
axs[0].set_xlabel('Distance')
axs[0].set_ylabel('Fare')
axs[0].set_title('All data')

idx = (df.distance < 10) & (df.fare_amount < 75)
axs[1].scatter(df[idx].distance, df[idx].fare_amount, alpha=0.2)
axs[1].set_xlabel('Distance')
axs[1].set_ylabel('Fare')
axs[1].set_title('Distance < 10 mile, fare < $75');


df.drop(df.loc[((df.pickup_latitude == 0) & (df.pickup_longitude == 0))&((df.dropoff_latitude != 0) & (df.dropoff_longitude != 0)) & (df.fare_amount == 0)].index, axis = 0, inplace = True)


df.drop(df.loc[((df.pickup_latitude != 0) & (df.pickup_longitude != 0))&((df.dropoff_latitude == 0) & (df.dropoff_longitude == 0)) & (df.fare_amount == 0)].index, axis = 0, inplace = True)


distance_check = df.loc[(df.distance > 200)&(df.fare_amount != 0)]
distance_check


distance_check.distance = distance_check.apply(lambda x: (x.fare_amount - 2.50) / 1.56, axis = 1)
df.update(distance_check)


df[(df.distance == 0) & (df.fare_amount == 0)]


df.drop(df[(df.distance == 0) & (df.fare_amount == 0)].index, axis = 0, inplace = True)


df.loc[(((df.hour >= 6) & (df.hour <= 20)) & ((df.day_of_week >= 1) & (df.day_of_week <= 5)) & (df.distance == 0) & (df.fare_amount < 2.5))]


df.drop(df.loc[(((df.hour >= 6) & (df.hour <= 20)) & ((df.day_of_week >= 1) & (df.day_of_week <= 5)) & 
                  (df.distance == 0) & (df.fare_amount < 2.5))].index, axis = 0, inplace = True)


df.loc[((df.day_of_week == 0) | (df.day_of_week == 6)) & (df.distance == 0) & (df.fare_amount < 2.5)]


df.drop(df.loc[((df.day_of_week == 0) | (df.day_of_week == 6)) & (df.distance == 0) & (df.fare_amount < 2.5)].index, axis = 0, inplace = True)


fare_check = df.loc[(df.fare_amount == 0) & (df.distance != 0)]
fare_check.sort_values('distance',ascending = False)


fare_check.fare_amount = fare_check.apply(lambda x: ((x.distance * 1.56) + 2.5), axis = 1)
df.update(fare_check)


dist_check = df.loc[(df.fare_amount > 3) & (df.distance == 0)]
dist_check.sort_values('fare_amount',ascending = False)


dist_check.distance = dist_check.apply(lambda x: ((x.distance - 2.5) / 1.56), axis = 1)
df.update(dist_check)


target_check = df.loc[(df.fare_amount < 100) & (df.distance > 100)]
target_check.sort_values(['distance','fare_amount'], ascending = False)


target_check.fare_amount = target_check.apply(lambda x: ((x.distance * 1.56) + 2.5), axis = 1)
df.update(target_check)


fig, ax = plt.subplots(1, 1, figsize=(16,10))
ax.scatter(x=df.distance, y=df.fare_amount, s=1.5)
ax.set_xlabel('Distance')
ax.set_ylabel('Fare')
ax.set_title('Fare Distribution with relation to distance')


bb = (min(test_df.pickup_longitude.min(), test_df.dropoff_longitude.min()), 
      max(test_df.pickup_longitude.max(), test_df.dropoff_longitude.max()), 
      min(test_df.pickup_latitude.min(), test_df.dropoff_latitude.min()), 
      max(test_df.pickup_latitude.max(), test_df.dropoff_latitude.max()))
bb


def filter_range(df, bb):
    return (df.pickup_longitude >= bb[0]) & (df.pickup_longitude <= bb[1]) & \
           (df.pickup_latitude >= bb[2]) & (df.pickup_latitude <= bb[3]) & \
           (df.dropoff_longitude >= bb[0]) & (df.dropoff_longitude <= bb[1]) & \
           (df.dropoff_latitude >= bb[2]) & (df.dropoff_latitude <= bb[3])


fig, axs = plt.subplots(1, 2, figsize=(24,12))
idx = filter_range(df, (-74.1, -73.7, 40.6, 40.9))
axs[0].set_title('Pickup Locations')
axs[0].scatter(df[idx].pickup_longitude, df[idx].pickup_latitude, c='r', s=0.01, alpha=0.5)
axs[1].set_title('Dropoff Locations')
axs[1].scatter(df[idx].dropoff_longitude, df[idx].dropoff_latitude, c='b', s=0.01, alpha=0.5)


plt.figure(figsize=(15,12))
cor = df.corr()
sns.heatmap(cor, annot=True, cmap=plt.cm.CMRmap_r)
plt.show()


distance(test_df, 'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude')


test_df['year'] = test_df.pickup_datetime.dt.year
test_df['month'] = test_df.pickup_datetime.dt.month
test_df['date'] = test_df.pickup_datetime.dt.day
test_df['day_of_week'] = test_df.pickup_datetime.dt.dayofweek
test_df['hour'] = test_df.pickup_datetime.dt.hour


df.drop(['key','pickup_datetime'], axis = 1, inplace = True)
test_df.drop(['key','pickup_datetime'], axis = 1, inplace = True)


train_df = ps.from_pandas(df)
train_df = train_df.to_spark()


feature_assembler = VectorAssembler(inputCols=['pickup_longitude', 'dropoff_longitude', 'passenger_count', 'distance', 'year', 'month', 'date'], outputCol="features")

lr = LinearRegression(labelCol="fare_amount")

pipeline = Pipeline(stages=[feature_assembler, lr])

train, test = train_df.randomSplit([0.75, 0.25])

lr_model = pipeline.fit(train)

predictions = lr_model.transform(test)

evaluator = RegressionEvaluator(labelCol= 'fare_amount', predictionCol= 'prediction')

print('RMSE:', evaluator.evaluate(predictions, {evaluator.metricName: "rmse"}))
print('R-squared:', evaluator.evaluate(predictions, {evaluator.metricName: "r2"}))


feature_assembler = VectorAssembler(inputCols=['pickup_longitude', 'dropoff_longitude', 'passenger_count', 'distance', 'year', 'month', 'day_of_week', 'hour'], outputCol="features")

dt = DecisionTreeRegressor(labelCol="fare_amount")

pipeline = Pipeline(stages=[feature_assembler, dt])

train, test = train_df.randomSplit([0.75, 0.25])

dt_model = pipeline.fit(train)

predictions = dt_model.transform(test)

evaluator = RegressionEvaluator(labelCol= 'fare_amount', predictionCol= 'prediction')

print('RMSE:', evaluator.evaluate(predictions, {evaluator.metricName: "rmse"}))
print('R-squared:', evaluator.evaluate(predictions, {evaluator.metricName: "r2"}))


feature_assembler = VectorAssembler(inputCols=['pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'passenger_count', 'distance', 'year', 'month', 'date', 'hour'], outputCol="features")

rf = RandomForestRegressor(labelCol="fare_amount")

pipeline = Pipeline(stages=[feature_assembler, rf])

train, test = train_df.randomSplit([0.75, 0.25])

rf_model = pipeline.fit(train)

predictions = rf_model.transform(test)

evaluator = RegressionEvaluator(labelCol= 'fare_amount', predictionCol= 'prediction')

print('RMSE:', evaluator.evaluate(predictions, {evaluator.metricName: "rmse"}))
print('R-squared:', evaluator.evaluate(predictions, {evaluator.metricName: "r2"}))


feature_assembler = VectorAssembler(inputCols=['pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'passenger_count', 'distance', 'year', 'month', 'date', 'day_of_week'], outputCol="features")

gbr = GBTRegressor(labelCol="fare_amount")

pipeline = Pipeline(stages=[feature_assembler, gbr])

train, test = train_df.randomSplit([0.75, 0.25])

gbr_model = pipeline.fit(train)

predictions = gbr_model.transform(test)

evaluator = RegressionEvaluator(labelCol= 'fare_amount', predictionCol= 'prediction')

print('RMSE:', evaluator.evaluate(predictions, {evaluator.metricName: "rmse"}))
print('R-squared:', evaluator.evaluate(predictions, {evaluator.metricName: "r2"}))


workspace_name = "fare-prediction"
workspace_location = "eastus"
resource_group = "NYC_Price_Prediction"
subscription_id = "75b2e7f9-9c16-4ece-9685-e7db922c1b62"   
authentication = InteractiveLoginAuthentication(tenant_id = "bc7e12a8-5234-4447-9371-44fc05a1d39c")

workspace = Workspace.create(name = workspace_name,
                             location = workspace_location,
                             resource_group = resource_group,
                             subscription_id = subscription_id,
                             auth = authentication,
                             exist_ok = True)


def fare_training(df, alpha, n_estimators):
    warnings.filterwarnings("ignore")
    if float(alpha) is None:
        alpha = 0.9
    else:
        alpha = float(alpha)
    
    if int(n_estimators) is None:
        n_estimators = 100
    else:
        n_estimators = int(n_estimators)
        
    np.random.seed(42)
    train, test = train_test_split(df)
    X_train = train.drop(["fare_amount"], axis=1)
    X_test = test.drop(["fare_amount"], axis=1)
    y_train = train[["fare_amount"]]
    y_test = test[["fare_amount"]]

    def evaluation_metrics(true, pred):
        return np.sqrt(mean_squared_error(true, pred)), mean_absolute_error(true, pred), r2_score(true, pred)
  
    with mlflow.start_run() as run:
        gbr = GradientBoostingRegressor(alpha=alpha, n_estimators = n_estimators, random_state = 42)
        gbr.fit(X_train, y_train)

        predictions = gbr.predict(X_test)

        (rmse, mae, r2) = evaluation_metrics(y_test, predictions)

        print("GradientBoostingRegressor(alpha=%f, n_estimators=%f):" % (alpha, n_estimators))
        print("  RMSE: %s" % rmse)
        print("  MAE: %s" % mae)
        print("  R2 Score: %s" % r2)

        mlflow.log_param("alpha", alpha)
        mlflow.log_param("n_estimators", n_estimators)
        
        mlflow.log_metric("rmse", rmse)
        mlflow.log_metric("mae", mae)
        mlflow.log_metric("r2", r2)
        
        mlflow.sklearn.log_model(gbr, "model")
        
        temp = int(random()*1000)
        path = "/dbfs/mlflow/fare_test/model-%f-%f-%f" % (alpha, n_estimators, temp)
        mlflow.sklearn.save_model(gbr, path)

        run_id = run.info.run_id
        print('Run ID: ', run_id)
        model_uri = "runs:/" + run_id + "/model"
        print('model_uri: ', model_uri)

    return run_id, model_uri


run_id, model_uri = fare_training(df, 0.01, 80)


model_image, azure_model = mlflow.azureml.build_image(model_uri = model_uri, 
                                                      workspace = workspace,
                                                      model_name = "gbr-model",
                                                      image_name = "gbr-model",
                                                      description="Gradient Boosting Regressor for fare prediction",
                                                      synchronous=False)
model_image.wait_for_creation(show_output=True)

	fare_amount	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count
count	5.500000e+06	5.500000e+06	5.500000e+06	5.499964e+06	5.499964e+06	5.500000e+06
mean	1.134239e+01	-7.250980e+01	3.992242e+01	-7.250923e+01	3.991838e+01	1.684539e+00
std	9.819104e+00	1.281230e+01	9.023598e+00	1.283150e+01	9.250279e+00	1.329587e+00
min	-1.000000e+02	-3.426609e+03	-3.488080e+03	-3.412653e+03	-3.488080e+03	0.000000e+00
25%	6.000000e+00	-7.399206e+01	4.073491e+01	-7.399140e+01	4.073404e+01	1.000000e+00
50%	8.500000e+00	-7.398181e+01	4.075263e+01	-7.398016e+01	4.075315e+01	1.000000e+00
75%	1.250000e+01	-7.396711e+01	4.076711e+01	-7.396368e+01	4.076811e+01	2.000000e+00
max	1.273310e+03	3.439426e+03	3.344459e+03	3.457622e+03	3.345917e+03	2.080000e+02

	key	fare_amount	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count
929022	2009-07-30 11:54:00	3.30	2009-07-30 11:54:00	0.000000	0.000000	0.000000	0.000000	208
1007609	2014-06-24 15:13:00	104.00	2014-06-24 15:13:00	-74.015780	40.715420	-74.170280	40.708340	9
2154045	2010-12-16 11:21:00	3.30	2010-12-16 11:21:00	0.000000	0.000000	0.000000	0.000000	208
2198549	2010-12-15 14:20:00	3.30	2010-12-15 14:20:00	0.000000	0.000000	0.000000	0.000000	208
2910347	2010-12-16 06:44:00	4.50	2010-12-16 06:44:00	0.000000	0.000000	0.000000	0.000000	208
3107489	2009-05-12 14:50:00	2.70	2009-05-12 14:50:00	-73.937818	40.758260	-73.937827	40.758250	208
3323791	2011-08-27 01:24:00	8.50	2011-08-27 01:24:00	-73.987858	40.760340	-74.000682	40.752413	129
4095440	2015-06-14 08:56:16	37.04	2015-06-14 08:56:16	-73.982094	40.756252	-73.872482	40.774506	9
4103745	2010-12-22 12:11:00	23.70	2010-12-22 12:11:00	0.000000	0.000000	0.000000	0.000000	208
4432483	2009-05-11 13:56:00	11.10	2009-05-11 13:56:00	-73.937733	40.758267	-73.937737	40.758273	208
4467314	2015-01-01 21:32:16	8.50	2015-01-01 21:32:16	-74.005867	40.740643	-73.988045	40.735100	7
4679603	2010-02-20 01:53:00	9.30	2010-02-20 01:53:00	0.554830	0.000000	-2069.478952	28.121107	51

	key	fare_amount	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count	distance	year	month	date	day_of_week	hour
472	2009-02-22 22:48:00	2.5	2009-02-22 22:48:00	0.000000	0.000000	-74.005433	40.726685	2	5385.945749	2009	2	22	6	22
1260	2011-03-10 20:25:00	5.7	2011-03-10 20:25:00	-73.973907	40.754743	0.000000	0.000000	2	5384.680323	2011	3	10	3	20
2280	2011-08-29 08:24:00	8.9	2011-08-29 08:24:00	-73.936667	40.757815	0.000000	40.757815	1	3744.920829	2011	8	29	0	8
2397	2012-06-24 17:11:10	45.0	2012-06-24 17:11:10	0.000000	0.000000	-74.010230	40.714553	2	5386.038620	2012	6	24	6	17
4278	2015-04-07 23:33:02	7.0	2015-04-07 23:33:02	-73.972702	40.757423	0.000000	0.000000	1	5384.652471	2015	4	7	1	23
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
5498033	2013-01-16 13:50:32	56.8	2013-01-16 13:50:32	0.000000	0.000000	-73.782726	40.648842	2	5373.483411	2013	1	16	2	13
5498148	2015-04-25 16:06:45	13.5	2015-04-25 16:06:45	0.000000	0.000000	-73.984329	40.737274	2	5384.994227	2015	4	25	5	16
5498288	2012-06-21 13:17:00	9.3	2012-06-21 13:17:00	-73.993742	40.759005	-73.993490	0.004457	2	2816.040848	2012	6	21	3	13
5498957	2012-07-13 13:50:00	7.7	2012-07-13 13:50:00	-7.978857	40.741195	-74.000840	40.739035	1	3369.164958	2012	7	13	4	13
5499385	2012-01-16 17:10:00	5.3	2012-01-16 17:10:00	0.000000	40.723482	-73.983288	40.725175	1	3749.152425	2012	1	16	0	17

	key	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count	year	month	date	day_of_week	hour
105051	2013-08-21 21:41:00	2013-08-21 21:41:00	0.000000	0.000000	0.000000	0.000000	2.0	2013.0	8.0	21.0	2.0	21.0
331597	2013-08-05 06:45:54	2013-08-05 06:45:54	-73.982354	40.679971	-73.982354	40.679971	1.0	2013.0	8.0	5.0	0.0	6.0
436658	2015-03-04 22:33:39	2015-03-04 22:33:39	-74.043442	40.788208	-74.043442	40.788208	1.0	2015.0	3.0	4.0	2.0	22.0
689250	2015-05-27 23:10:03	2015-05-27 23:10:03	-73.994125	40.741131	-73.994125	40.741131	1.0	2015.0	5.0	27.0	2.0	23.0
1221438	2015-02-02 21:58:28	2015-02-02 21:58:28	0.000000	0.000000	0.000000	0.000000	2.0	2015.0	2.0	2.0	0.0	21.0
1598058	2013-08-06 11:12:00	2013-08-06 11:12:00	0.000000	0.000000	0.000000	0.000000	2.0	2013.0	8.0	6.0	1.0	11.0
1662545	2010-03-13 02:39:29	2010-03-13 02:39:29	-74.035278	40.738683	-74.035278	40.738683	2.0	2010.0	3.0	13.0	5.0	2.0
2214498	2010-02-03 11:44:07	2010-02-03 11:44:07	-74.014147	40.710546	-74.014147	40.710546	1.0	2010.0	2.0	3.0	2.0	11.0
2329503	2015-02-19 09:43:11	2015-02-19 09:43:11	-73.953857	40.813251	-73.953857	40.813251	1.0	2015.0	2.0	19.0	3.0	9.0
2581417	2015-05-07 19:12:43	2015-05-07 19:12:43	-73.913727	40.779018	-73.913727	40.779018	1.0	2015.0	5.0	7.0	3.0	19.0
2618454	2015-03-26 05:43:22	2015-03-26 05:43:22	0.000000	0.000000	0.000000	0.000000	1.0	2015.0	3.0	26.0	3.0	5.0
2991619	2015-01-15 14:49:08	2015-01-15 14:49:08	0.000000	0.000000	0.000000	0.000000	1.0	2015.0	1.0	15.0	3.0	14.0
3070651	2015-01-25 20:54:02	2015-01-25 20:54:02	-73.981293	40.769012	-73.981293	40.769012	1.0	2015.0	1.0	25.0	6.0	20.0
3252539	2015-01-14 10:15:51	2015-01-14 10:15:51	-73.937508	40.758129	-73.937508	40.758129	1.0	2015.0	1.0	14.0	2.0	10.0
3712453	2010-03-29 12:47:22	2010-03-29 12:47:22	-73.836776	40.869879	-73.836776	40.869879	1.0	2010.0	3.0	29.0	0.0	12.0
4191567	2010-02-22 01:02:16	2010-02-22 01:02:16	-73.981984	40.757467	-73.981984	40.757467	1.0	2010.0	2.0	22.0	0.0	1.0
4307073	2010-02-13 00:30:23	2010-02-13 00:30:23	-73.968945	40.764540	-73.968945	40.764540	1.0	2010.0	2.0	13.0	5.0	0.0
4322959	2015-03-25 07:20:55	2015-03-25 07:20:55	0.000000	0.000000	0.000000	0.000000	5.0	2015.0	3.0	25.0	2.0	7.0
4377466	2015-01-16 14:56:36	2015-01-16 14:56:36	-73.937531	40.758289	-73.937531	40.758289	1.0	2015.0	1.0	16.0	4.0	14.0
4383356	2013-08-17 13:55:00	2013-08-17 13:55:00	0.000000	0.000000	0.000000	0.000000	2.0	2013.0	8.0	17.0	5.0	13.0
4819787	2015-02-18 06:47:03	2015-02-18 06:47:03	-73.937820	40.758190	-73.937820	40.758190	1.0	2015.0	2.0	18.0	2.0	6.0
4851000	2010-02-10 20:18:01	2010-02-10 20:18:01	-74.003095	40.743403	-74.003095	40.743403	1.0	2010.0	2.0	10.0	2.0	20.0
5406596	2015-04-21 03:41:29	2015-04-21 03:41:29	-73.937439	40.758160	-73.937439	40.758160	1.0	2015.0	4.0	21.0	1.0	3.0

	key	fare_amount	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count	year	month	date	day_of_week	hour
830616	2015-01-21 06:29:21	0.01	2015-01-21 06:29:21	0.000000	0.000000	0.000000	0.000000	1.0	2015.0	1.0	21.0	2.0	6.0
925892	2015-01-21 14:12:00	1.90	2015-01-21 14:12:00	0.000000	0.000000	0.000000	0.000000	1.0	2015.0	1.0	21.0	2.0	14.0
1112195	2015-03-25 14:55:38	1.60	2015-03-25 14:55:38	0.000000	0.000000	0.000000	0.000000	2.0	2015.0	3.0	25.0	2.0	14.0
1215355	2015-05-29 18:15:27	2.00	2015-05-29 18:15:27	-73.903214	40.747318	-73.903214	40.747318	1.0	2015.0	5.0	29.0	4.0	18.0
2097377	2013-08-30 15:11:00	0.50	2013-08-30 15:11:00	-121.340000	38.903350	-121.340000	38.903350	1.0	2013.0	8.0	30.0	4.0	15.0
2322071	2010-02-25 12:44:00	1.25	2010-02-25 12:44:00	-73.937722	40.758318	-73.937722	40.758318	5.0	2010.0	2.0	25.0	3.0	12.0
2949904	2015-01-22 13:34:00	2.00	2015-01-22 13:34:00	0.000000	0.000000	0.000000	0.000000	1.0	2015.0	1.0	22.0	3.0	13.0
3010346	2015-01-23 08:50:38	0.01	2015-01-23 08:50:38	0.000000	0.000000	0.000000	0.000000	1.0	2015.0	1.0	23.0	4.0	8.0
4112515	2015-04-15 20:12:10	0.01	2015-04-15 20:12:10	-73.994553	40.750240	-73.994553	40.750240	1.0	2015.0	4.0	15.0	2.0	20.0
4920415	2013-08-09 18:39:00	1.50	2013-08-09 18:39:00	0.000000	0.000000	0.000000	0.000000	1.0	2013.0	8.0	9.0	4.0	18.0
5425695	2015-02-05 17:02:10	0.01	2015-02-05 17:02:10	-73.866501	40.770050	-73.866501	40.770050	1.0	2015.0	2.0	5.0	3.0	17.0

Import the libraries¶

Mount the Azure Storage¶

Data Visualization¶

Feature Engineering¶

Modeling¶

Deployment¶

	key	fare_amount	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count	distance	year	month	date	day_of_week	hour
3614406	2015-05-12 08:28:24	0.0	2015-05-12 08:28:24	-73.820679	40.665684	-74.181442	40.687679	1.0	18.966283	2015.0	5.0	12.0	1.0	8.0
5325349	2015-02-07 01:58:09	0.0	2015-02-07 01:58:09	-74.009926	40.736710	-74.282852	40.614143	3.0	16.621915	2015.0	2.0	7.0	5.0	1.0
1107618	2015-05-31 01:52:15	0.0	2015-05-31 01:52:15	-73.997482	40.758915	-74.144936	40.946930	4.0	15.105223	2015.0	5.0	31.0	6.0	1.0
3763201	2010-03-05 21:43:42	0.0	2010-03-05 21:43:42	-73.786049	40.641929	-73.981236	40.782779	4.0	14.114868	2010.0	3.0	5.0	4.0	21.0
1902135	2010-03-08 20:23:36	0.0	2010-03-08 20:23:36	-73.786727	40.640482	-73.979946	40.762910	5.0	13.191238	2010.0	3.0	8.0	0.0	20.0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
211499	2010-03-08 18:22:00	0.0	2010-03-08 18:22:00	-73.937353	40.758227	-73.937358	40.758230	1.0	0.000334	2010.0	3.0	8.0	0.0	18.0
2771692	2015-05-31 04:40:53	0.0	2015-05-31 04:40:53	-73.896584	40.752499	-73.896584	40.752502	1.0	0.000264	2015.0	5.0	31.0	6.0	4.0
495273	2015-05-26 12:28:03	0.0	2015-05-26 12:28:03	-73.986473	40.721909	-73.986473	40.721905	1.0	0.000264	2015.0	5.0	26.0	1.0	12.0
670254	2015-02-22 19:26:37	0.0	2015-02-22 19:26:37	-73.994370	40.750725	-73.994370	40.750721	1.0	0.000264	2015.0	2.0	22.0	6.0	19.0
4942824	2013-08-20 13:54:39	0.0	2013-08-20 13:54:39	-74.002551	40.733974	-74.002555	40.733975	6.0	0.000221	2013.0	8.0	20.0	1.0	13.0

	key	fare_amount	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count	distance	year	month	date	day_of_week	hour
1144706	2015-03-09 15:12:00	1273.31	2015-03-09 15:12:00	0.000000	0.000000	0.000000	0.000000	1.0	0.0	2015.0	3.0	9.0	0.0	15.0
3723759	2015-03-10 09:48:00	698.16	2015-03-10 09:48:00	0.000000	0.000000	0.000000	0.000000	1.0	0.0	2015.0	3.0	10.0	1.0	9.0
101885	2011-09-12 09:33:56	500.00	2011-09-12 09:33:56	-73.965961	40.640595	-73.965961	40.640595	1.0	0.0	2011.0	9.0	12.0	0.0	9.0
3770478	2013-01-21 03:57:00	300.25	2013-01-21 03:57:00	-73.990137	40.756872	-73.990137	40.756872	1.0	0.0	2013.0	1.0	21.0	0.0	3.0
2427679	2015-01-03 03:57:26	300.00	2015-01-03 03:57:26	-73.969688	40.592640	-73.969688	40.592640	1.0	0.0	2015.0	1.0	3.0	5.0	3.0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
4828186	2011-05-12 09:39:20	3.30	2011-05-12 09:39:20	-73.983423	40.744259	-73.983423	40.744259	1.0	0.0	2011.0	5.0	12.0	3.0	9.0
366224	2009-07-30 10:05:21	3.30	2009-07-30 10:05:21	-73.137393	41.366138	-73.137393	41.366138	1.0	0.0	2009.0	7.0	30.0	3.0	10.0
4125935	2011-08-02 16:57:00	3.30	2011-08-02 16:57:00	-73.982505	40.735358	-73.982505	40.735358	2.0	0.0	2011.0	8.0	2.0	1.0	16.0
3841998	2013-05-22 10:28:00	3.27	2013-05-22 10:28:00	0.000000	0.000000	0.000000	0.000000	1.0	0.0	2013.0	5.0	22.0	2.0	10.0
4724757	2013-12-13 08:36:00	3.25	2013-12-13 08:36:00	0.000000	0.000000	0.000000	0.000000	1.0	0.0	2013.0	12.0	13.0	4.0	8.0

	key	fare_amount	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count	distance	year	month	date	day_of_week	hour
1632032	2013-04-21 15:13:00	11.0	2013-04-21 15:13:00	-74.002510	40.734320	-70.183332	40.756540	1.0	199.922210	2013.0	4.0	21.0	6.0	15.0
2811044	2012-03-22 08:34:00	5.3	2012-03-22 08:34:00	-74.005112	40.741367	-77.816665	40.732262	1.0	199.544089	2012.0	3.0	22.0	3.0	8.0
1357154	2010-05-11 16:24:42	10.9	2010-05-11 16:24:42	-70.961200	42.505300	-73.967621	40.762009	1.0	196.489427	2010.0	5.0	11.0	1.0	16.0
4822280	2013-09-02 20:28:00	8.5	2013-09-02 20:28:00	-73.983740	40.734302	-77.716667	40.743877	1.0	195.421917	2013.0	9.0	2.0	0.0	20.0
1738206	2013-01-10 14:47:00	7.5	2013-01-10 14:47:00	-77.633333	40.784112	-73.967712	40.768627	2.0	191.793184	2013.0	1.0	10.0	3.0	14.0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
1926509	2011-03-11 21:45:31	49.8	2011-03-11 21:45:31	-73.963062	39.629601	-75.342620	40.643574	1.0	101.090299	2011.0	3.0	11.0	4.0	21.0
3286123	2011-03-28 15:43:49	49.8	2011-03-28 15:43:49	-75.329236	40.645135	-73.958087	39.627681	2.0	100.938013	2011.0	3.0	28.0	0.0	15.0
4507221	2014-09-16 04:51:00	11.5	2014-09-16 04:51:00	-73.980398	40.734552	-75.468602	41.670472	1.0	100.834201	2014.0	9.0	16.0	1.0	4.0
1937753	2015-05-08 13:51:46	20.5	2015-05-08 13:51:46	-72.063202	40.834679	-73.982452	40.772621	2.0	100.473338	2015.0	5.0	8.0	4.0	13.0
2234097	2011-03-27 21:34:02	49.8	2011-03-27 21:34:02	-75.343970	40.645350	-73.933972	39.671217	1.0	100.371781	2011.0	3.0	27.0	6.0	21.0

key	fare_amount	pickup_datetime	pickup_longitude	pickup_latitude	dropoff_longitude	dropoff_latitude	passenger_count
2009-06-15T17:26:21.000+0000	4.5	2009-06-15T17:26:21.000+0000	-73.844311	40.721319	-73.84161	40.712278	1
2010-01-05T16:52:16.000+0000	16.9	2010-01-05T16:52:16.000+0000	-74.016048	40.711303	-73.979268	40.782004	1
2011-08-18T00:35:00.000+0000	5.7	2011-08-18T00:35:00.000+0000	-73.982738	40.76127	-73.991242	40.750562	2
2012-04-21T04:30:42.000+0000	7.7	2012-04-21T04:30:42.000+0000	-73.98713	40.733143	-73.991567	40.758092	1
2010-03-09T07:51:00.000+0000	5.3	2010-03-09T07:51:00.000+0000	-73.968095	40.768008	-73.956655	40.783762	1
2011-01-06T09:50:45.000+0000	12.1	2011-01-06T09:50:45.000+0000	-74.000964	40.73163	-73.972892	40.758233	1
2012-11-20T20:35:00.000+0000	7.5	2012-11-20T20:35:00.000+0000	-73.980002	40.751662	-73.973802	40.764842	1
2012-01-04T17:22:00.000+0000	16.5	2012-01-04T17:22:00.000+0000	-73.9513	40.774138	-73.990095	40.751048	1
2012-12-03T13:10:00.000+0000	9.0	2012-12-03T13:10:00.000+0000	-74.006462	40.726713	-73.993078	40.731628	1
2009-09-02T01:11:00.000+0000	8.9	2009-09-02T01:11:00.000+0000	-73.980658	40.733873	-73.99154	40.758138	2
2012-04-08T07:30:50.000+0000	5.3	2012-04-08T07:30:50.000+0000	-73.996335	40.737142	-73.980721	40.733559	1
2012-12-24T11:24:00.000+0000	5.5	2012-12-24T11:24:00.000+0000	0.0	0.0	0.0	0.0	3
2009-11-06T01:04:03.000+0000	4.1	2009-11-06T01:04:03.000+0000	-73.991601	40.744712	-73.983081	40.744682	2
2013-07-02T19:54:00.000+0000	7.0	2013-07-02T19:54:00.000+0000	-74.00536	40.728867	-74.008913	40.710907	1
2011-04-05T17:11:05.000+0000	7.7	2011-04-05T17:11:05.000+0000	-74.001821	40.737547	-73.99806	40.722788	2
2013-11-23T12:57:00.000+0000	5.0	2013-11-23T12:57:00.000+0000	0.0	0.0	0.0	0.0	1
2014-02-19T07:22:00.000+0000	12.5	2014-02-19T07:22:00.000+0000	-73.98643	40.760465	-73.98899	40.737075	1
2009-07-22T16:08:00.000+0000	5.3	2009-07-22T16:08:00.000+0000	-73.98106	40.73769	-73.994177	40.728412	1
2010-07-07T14:52:00.000+0000	5.3	2010-07-07T14:52:00.000+0000	-73.969505	40.784843	-73.958732	40.783357	1
2014-12-06T20:36:22.000+0000	4.0	2014-12-06T20:36:22.000+0000	-73.979815	40.751902	-73.979446	40.755481	1
2010-09-07T13:18:00.000+0000	10.5	2010-09-07T13:18:00.000+0000	-73.985382	40.747858	-73.978377	40.76207	1
2013-02-12T12:15:46.000+0000	11.5	2013-02-12T12:15:46.000+0000	-73.957954	40.779252	-73.96125	40.758787	1
2009-08-06T18:17:23.000+0000	4.5	2009-08-06T18:17:23.000+0000	-73.991707	40.770505	-73.985459	40.763671	1
2010-12-06T12:29:00.000+0000	4.9	2010-12-06T12:29:00.000+0000	-74.000632	40.747473	-73.986672	40.740577	1
2009-12-10T15:37:00.000+0000	6.1	2009-12-10T15:37:00.000+0000	-73.969622	40.756973	-73.981152	40.759712	1
2011-06-21T16:15:00.000+0000	7.3	2011-06-21T16:15:00.000+0000	-73.991875	40.754437	-73.97723	40.774323	3
2011-02-07T20:01:00.000+0000	6.5	2011-02-07T20:01:00.000+0000	0.0	0.0	0.0	0.0	1
2011-06-28T19:47:00.000+0000	4.5	2011-06-28T19:47:00.000+0000	-73.988893	40.76016	-73.986445	40.757857	3
2012-05-04T06:11:20.000+0000	9.3	2012-05-04T06:11:20.000+0000	-73.989258	40.690835	-74.004133	40.72569	1
2013-08-11T00:52:00.000+0000	4.5	2013-08-11T00:52:00.000+0000	-73.98102	40.73776	-73.980668	40.730497	2
2014-02-19T16:03:00.000+0000	5.5	2014-02-19T16:03:00.000+0000	-73.976075	40.752422	-73.981082	40.759285	1
2015-06-21T21:46:34.000+0000	22.54	2015-06-21T21:46:34.000+0000	-74.01048278808595	40.71766662597656	-73.98577117919923	40.66036605834961	1
2009-01-09T16:10:00.000+0000	31.9	2009-01-09T16:10:00.000+0000	-73.873027	40.773883	-73.984545	40.769545	3
2011-09-07T14:05:00.000+0000	5.7	2011-09-07T14:05:00.000+0000	-73.976162	40.744988	-73.990002	40.738323	2
2011-09-07T23:05:44.000+0000	18.1	2011-09-07T23:05:44.000+0000	-73.993717	40.740785	-73.999272	40.676617	1
2015-01-11T03:15:38.000+0000	58.0	2015-01-11T03:15:38.000+0000	-73.98332977294923	40.73871994018555	-73.93319702148438	40.84722518920898	1
2015-03-12T08:08:38.000+0000	4.5	2015-03-12T08:08:38.000+0000	-73.99017333984375	40.756446838378906	-73.9856185913086	40.7628288269043	1
2012-01-19T10:15:30.000+0000	5.3	2012-01-19T10:15:30.000+0000	-73.995199	40.75474	-74.005416	40.751084	1
2013-11-24T11:42:00.000+0000	9.0	2013-11-24T11:42:00.000+0000	-73.95774	40.773617	-73.977087	40.758465	6
2009-03-02T20:42:49.000+0000	9.8	2009-03-02T20:42:49.000+0000	-73.972673	40.759186	-73.969897	40.791367	1